#!/bin/sh
# Copyright (c) 2013, bkil.hu
# This program is free software and can be distributed under the terms of
# the GNU General Public License v2,
# see COPYING for detailed licensing terms.

disclaimer_text() {
 cat << EOF
DISCLAIMER

This is not exhaustive legal information, please consult a jurist if in doubt.

I wrote this application for personal entertainment.
It downloads less than 1MB of gzip'ped HTML from a crowdsourced aggregating
site that lists basic information about TV and radio channels relevant to my
area.
The script then runs clever transformations on the HTML table set to arrive at
a unified CSV (comma separated values) table.
Note that for a given locality, you could effortlessly get the exact same
data set by scanning using a motorized dish, and parsing each digital stream's
metadata.
I can then import the table to LibreOffice Base and run funky queries on it
like:

select * from lyng where ("min EIRP" >= 50) AND (("Encryption" IS NULL) OR ("Encryption"='freetv')) AND (("System"='DVB-S2') OR ("Codec" IS NOT NULL) AND ("Codec"!='MPEG-2')) order by "Channel Name";

Note that mechanically assembled, complete lists of facts without any added
value can not by default be considered for copyright protection.
However, there could exist jurisdictions which recognize a special, so called
"database right".
To familiarize yourself with the concept, please refer to relevant articles:
http://en.wikipedia.org/wiki/Database_right#United_States
http://en.wikipedia.org/wiki/Database_Directive#Sui_generis_right
For example, works produced in the USA do not even enjoy database rights.

In the worst case, the resulting derived data set will fall under the same
copyright as the original and will be owned by the original author.
You must obey the original restrictions and not copy or commercially use any
part of the resulting data in this case.

I'm neither affiliated with, nor wish to harm the data provider lyngsat.com.
It was a default choice of mine because it was the first such site I found.
I could adapt to flysat.com, kingofsat.net, satbeams.com, tracksat.com, etc.
Although, I'm not obliged to do so, I'm preserving the source field in the CSV
to give credit to the original spotters in good faith.

Your continued use of this program must be respecting your local laws as
stated above.
EOF
}

main() {
 greet
 grab_main
}

greet() {
 local CONF=$HOME/.`basename $0`.conf
 local AGREE=""
 [ -f "$CONF" ] && . "$CONF"
 if [ ! "$AGREE" = "1" ]; then
  disclaimer_text
  read -p "Type uppercase YES to accept: " YES
  if [ ! "$YES" = "YES" ]; then
   echo "DECLINE" >&2
   exit 1
  fi
  echo "AGREE=1" >> $CONF
 fi
}

grab_main() {
 FS=";"
 CS=","

 grab_all > lyng.csv

 {
  echo "DEBUG lines:"
  sed -n '1p;2p;52p' lyng.csv
 } >&2
}

grab_all() {
 local SITE="www.lyngsat.com"
 grep '<td width=70 ' `wget2 "http://$SITE/europe.html"` |
 sed -r 's~^.*<a href="([^"]*)">.*$~\1~' |
 while read URL
 do
  process_single < `wget2 "$URL"`
 done |
 process_combined
 # rm -R "$SITE" # DEBUG
}

wget2() {
 local FILE=`echo "$1" | sed "s~^http://~~"`
 echo "$FILE"
 if [ ! -f "$FILE" ]; then
  wget --force-directories --no-clobber --wait 5 "$@" >&2
  sleep 5
 fi
}

process_single() {
 iconv -f iso-8859-1 -t utf-8 |
 html2tag |
 tags2csv
}

process_combined() {
 adjustheader |
# less; exit
 splitcsvcols |
 prune_cols "TpX" "max EIRP" |
 fix_date |
 assimilate_scpc |
 decode_codec |
 prune_cols "CLogo" "PLogo" "PInfo" "SR-FEC0" \
  "SID2" "VPID2" "APID Lang.2" "Codec,Encryption2" |
 string_cols
}

html2tag() {
 local FS0=">"

 grep -E '<tr>|<td[ >]|<title>' |
 sed -rn '2,/<td colspan=10 align="center">/ ! p' |
 sed -rn '/Colour codes on this satellite chart:/,$ ! p' |
 grep -vE '<td colspan=10 align="center">' |
 sed -r "
  s~&nbsp;~ ~g
  s~^<title>(.*) at ([1-9][0-9]*(|\.[0-9]))[^EW]*E - LyngSat</title>$~\1${FS0}\2~
  s~^<title>(.*) at ([1-9][0-9]*(|\.[0-9]))[^EW]*W - LyngSat</title>$~\1${FS0}-\2~
  s~<a href=\"http://www.lyngsat.com/freetv/[^\"]*\">~freetv<br>~
  s~<a href=\"http://www.lyngsat.com/freeradio/[^\"]*\">~freeradio<br>~
  s~<a href=\"(http://www.lyngsat-stream.com/tvchannels/[^\">]*)\">~\1~
  s~<tr>~0${FS0}~
  s~<br>~${CS}~g
  s~<td [^>]*\<rowspan=([0-9]+)(| [^>]*)>~\1${FS0}~
  s~(.)(<td[ >])~\1\n\2~g
  s~<td [^>]*\<colspan=2(| [^>]*)>~1${FS0}\n&~g
  s~<td [^>]*\<colspan=3(| [^>]*)>~1${FS0}\n1${FS0}\n&~g" |
 sed -r "
  s~<td(| )~1${FS0}&~
  s~<[^>]*>~ ~g
  s~ *${CS} *~${CS}~g
  s~ *${FS0} *~${FS0}~g
  s~ +~ ~g
  s~${CS}+~${CS}~g
  s~^([^${FS0}]*${FS0}) *${CS}?~\1~
  s~${CS}? *$~~
  s~;~\&#59<>~g
  s~${FS0}~${FS}~g
 "
}

tags2csv() {
 awk -F $FS '
  BEGIN{
   OFS = FS;
   col = 0;
   rep[0] = 0;
   ncol = 0;
   tp = 0;
  }
  {
   if (NR == 1) {
    satname = $1;
    satpos = $2;
   } else if ($1 == 0) {
    consolidate();
   } else {
    while ((col<ncol) && (rep[col]>0)) {
     rep[col]--;
     col++;
    }
    if (col==0) {
     tp = 1;
    }
    rep[col] = $1 - 1;
    cols[col] = $2;
    col++;
    if (col>ncol) {
     ncol = col;
    }
   }
  }
 END {
  consolidate();
 }
 function consolidate() {
   if (ncol == 0) {
# first run
    return;
   } else if (col == 0) {
# theoretical duplicate table row
    return;
   } else {
    if (header[0] == "") {
     header[0] = cols[0];
    }
    while (col<ncol) {
     if (rep[col]==0) {
      cols[col] = "#MISSING"
     }
     rep[col]--;
     col++;
    }
    col = 0;
    if (tp == 0) {
     write();
    } else {
     tp = 0;
     nmeta = 7;
     for (i=0; i<nmeta; i++) {
      meta[i] = cols[i+1];
      if ((rep[0] == 0) && (cols[0] != header[0])) {
       cols[i+1] = "";
      }
     }
     if (rep[0] == 0) {
      write();
     }
    }
   }
 }
 function write() {
  for (i=0; i<ncol; i++) {
   printf cols[i] FS;
   if (rep[i] == 0) { # extra sanity filler
    cols[i] = "#BLANK";
   }
  }
  i = 0
  for (; i<nmeta; i++) {
   printf meta[i] FS;
  }
  print satpos;
 }
 '
}

adjustheader() {
 echo "Freq. Pol.,TpX Tp;CLogo;Channel Name;CInfo;Codec,Encryption;SID;VPID;APID Lang.;Beam,min EIRP-max EIRP,C/N lock;Source,Updated;PLogo;Provider Name;PInfo;System,Codec,Encryption2;SR-FEC0;SR-FEC,MOD,SID2-VPID2;ONID-TID,APID Lang.2;Sat. Pos."
 fgrep -v "Freq.,Tp;;Provider Name,Channel Name;;System,Encryption;;SR-FEC,SID -VPID;ONID-TID,APID Lang.;Beam,EIRP (dBW),C/N lock;Source,Updated;;Provider Name,Channel Name;;System,Encryption;;SR-FEC,SID -VPID;ONID-TID,APID Lang."
}

splitcsvcols() {
 split_head 16 $CS |
 split_head 16 - |
 split_head 15 - |
 split_head 16 $CS |
 split_head 17 $CS |
 split_head 18 - |
 split_head 13 $CS |
 split_tail 9 $CS |
 split_head 8 $CS |
 split_head 9 $CS |
 split_head 9 - |
 split_head 0 $CS |
 split_head 1 " " |
 split_head 0 " " |
 sed "
  s~ *${CS} *~${CS}~g
  s~ *${FS} *~${FS}~g
 "
}

assimilate_scpc() {
 awk -F $FS '
  BEGIN {
   OFS = FS;
  }
  {
   if ($8 == "") {
    $8 = $25; # SID = SID2
   }
   if ($9 == "") {
    $9 = $26; # VPID = VPID2
   }
   if ($10 == "") {
    $10 = $29; # APID Lang. = APID Lang.2
   }
   if ($7 == "") {
    $7 = $20; # System,Encryption2 = System,Encryption2
    $20=""; #TODO: why are some clear transponders marked pay?
   }
   if ($12 == "") {
    $12 = 0; # min EIRP
   }
   print
  }
 '
}

split_head() {
# 1,2,3 -> 1;2,3
 local COL="$1" # zero-based
 local CH="$2"
 sed -r "
  s~^(([^${FS}]*${FS}){${COL}})(([^${FS}${CH}]*)|(([^${FS}${CH}]*)${CH}([^${FS}]*)))(|${FS}.*)$~\1\4\6${FS}\7\8~
 "
}

split_tail() {
# 1,2,3 -> 1,2;3
 local COL="$1" # zero-based
 local CH="$2"
 sed -r "
  s~^(([^${FS}]*${FS}){${COL}})(([^${FS}${CH}]*)|(([^${FS}]*)${CH}([^${FS}${CH}]*)))(|${FS}.*)$~\1\4\6${FS}\7\8~
 "
}

fix_date() {
# col 15 ("Updated"): 101231 -> 2010-12-31
 sed -r "
  s~^(([^${FS}]*${FS}){14})([0-1][0-9])(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[01])(|${FS}.*)$~\120\3-\4-\5\6~
 "
}

prune_cols() {
 read HEADER
 local FIELDS
 FIELDS=`echo "$HEADER"|sed "s~${FS}~\n~g"`
 local IDXPRUNE=""
 for COL in "$@"; do
  I=`echo "$FIELDS" |
     grep -n "$COL" |
     sed -r "s~^([0-9]+):.*$~\1~"`
  IDXPRUNE="${IDXPRUNE},$I"
 done
 IDXPRUNE=`echo "$IDXPRUNE" | sed "s~^,~~"`
 {
  echo $HEADER
  cat
 } |
 cut -d "$FS" --complement -f "$IDXPRUNE"
}

decode_codec() {
 # MPEG-4/HD -> MPEG-4/HD;
 # Viaccess 4.0 -> ;Viaccess 4.0
 # freetv -> ;freetv
 # freetv,MPEG-4/HD -> MPEG-4/HD;freetv
 # MPEG-2,Nagravision 3,Irdeto 2 -> MPEG-4/HD;Nagravision 3, Irdeto 2
 # -> ;
 sed -r "
  s~^(([^${FS}]*${FS}){6})~&${FS}~
  s~^(([^${FS}]*${FS}){6})${FS}(|[^${FS}]*${CS})(MPEG-[24][^${CS}${FS}]*)(|${CS}[^${FS}]*)(|${FS}.*)$~\1\4${FS}\3\5\6~
  s~${CS}*${FS}${CS}*~${FS}~g
  1 s~^(|.*;);Codec,Encryption~\1Codec;Encryption~
 "
}

string_cols() {
 cat
}

main "$@"

